from BlogElements import BlogProfile;
from BlogElements import BlogPosting;
import os;
import time;
import logging;

def getValue(line, name):
    tokens = line.split();
    value = line.replace(name + ':', '');
    value = value.strip();
    return value;

dateFormat1 = '%a, %d %b %Y %H:%M:%S %Z';
dateFormat2 = "%Y-%m-%d-%H-%M";

class PostingFile:
    def __init__(self):
        self.file = '';
    
    def open(self, filePath):
        if not os.path.exists(filePath):
            logging.error('no file ' + filePath);
            return -1;
        self.file = open(filePath);
        self.linecnt = 0;
        return 0;

    def seek(self, pos):
        self.file.seek(pos);
        
    def next(self):
        line = self.file.readline();
        self.linecnt += 1;
        if not line:
            return -1;
        self.entry = BlogPosting();
        
        status = '';
        while line:
            line = line.strip();
            if line.startswith('-' * 50):
                status = '';
                if self.entry.url <> '':
                    return 0;
            if line.startswith('TITLE:'):
                self.entry.title = getValue(line, 'TITLE');
                status = 'TITLE';
            elif line.startswith('URL:'):
                self.entry.url= getValue(line, 'URL');
                status = 'URL';
            elif line.startswith('DATE:'):
                datestr = getValue(line, 'DATE');
                try:
                    self.entry.date = time.mktime(time.strptime(datestr, dateFormat1));
                except:
                    try:
                        self.entry.date = time.mktime(time.strptime(datestr, dateFormat2));
                    except:
                        pass;
                status = 'DATE';
            elif line.startswith('DES:'):
                self.entry.description = getValue(line, 'DES');
                status = 'DES';
            elif line.startswith('COURL:'):
                self.entry.commentUrl = getValue(line, 'COURL');
                status = 'COURL';
            else:
                if status == 'DES':
                    self.entry.description += '\n' + line;
            line = self.file.readline();
            self.linecnt += 1;
        return -1;
    
    def getCurrElement(self):
        return self.entry;
        
    def close(self):
        if self.file:
            self.file.close();
        return 0;


class PostingFileFactory:
    def createFile(self):
        file = PostingFile();
        return file;

            
class ProfileFile:
    def __init__(self):
        self.file = '';
        
    def open(self, path):
        if not os.path.exists(path):
            return -1;
        self.file = open(path);
        self.linecnt = 1;
        return 0;
        
    def next(self):
        line = self.file.readline();
        self.linecnt += 1;
        if not line:
            return -1;
        self.profile = BlogProfile();
        
        while line:
            line = line.strip();
            if line.startswith('-' * 50):
                return 0;
            elif line.startswith('URL:'):
                self.profile.url = getValue(line, 'URL');
                status = 'url';
            elif line.startswith('AUTHOR:'):
                self.profile.author = getValue(line, 'AUTHOR');
                status = 'author';
            elif line.startswith('TITLE:'):
                self.profile.title = getValue(line, 'TITLE');
                status = 'title';
            elif line.startswith('AGE:'):
                self.profile.age = int(getValue(line, 'AGE'));
                status = 'age';
            elif line.startswith('REALNAME:'):
                self.profile.realName = getValue(line, 'REALNAME');
                status = 'realname';
            elif line.startswith('INTERESTS:'):
                self.profile.interests = getValue(line, 'INTERESTS');
                status = 'interests';
            elif line.startswith('HOMECOUNTRY:'):
                self.profile.homeCountry = getValue(line, 'HOMECOUNTRY');
                status = 'homecountry';
            elif line.startswith('ID:'):
                self.profile.userid = getValue(line, 'ID');
                status = 'id';
            elif line.startswith('GENDER:'):
                self.profile.gender = getValue(line, 'GENDER');
                status = 'gender';
            elif line.startswith('OCCUPATION:'):
                self.profile.occupation = getValue(line, 'OCCUPATION');
                status = 'occupation';
            elif line.startswith('HOMECITY:'):
                self.profile.homeCity = getValue(line, 'HOMECITY');
                status = 'homecity';
            elif line.startswith('PLACELIVED:'):
                self.profile.placesLived = getValue(line, 'PLACELIVED');
                status = 'placelived';
            elif line.startswith('MARITALSTATUS:'):
                self.profile.maritalStatus = getValue(line, 'MARITALSTATUS');
                status = 'maritalstatus';
            elif line.startswith('BIO:'):
                self.profile.bio = getValue(line, 'BIO');
                status = 'bio';
            elif line.startswith('TYPE:'):
                self.profile.type = getValue(line, 'TYPE');
                status = 'type';
            elif line:
                if status == 'bio':
                    self.profile.bio += line;
                else:
                    logging.warn('Unrecognize tag at line' + str(self.linecnt) + ':' + line);
            line = self.file.readline();
            self.linecnt += 1;
            if self.linecnt >= 516:
                iii = 0;
        return -1;
            
    def getCurrElement(self):
        return self.profile;
    
    def close(self):
        if self.file:
            self.file.close();
        return 0;

class ProfileFileFactory:
    def createFile(self):
        file = ProfileFile();
        return file;
    
def display(file, displayNum):
    succ = file.next();
    while succ <> -1:
        if succ >= 0:
            element = file.getCurrElement();
            print element;
        succ = file.next();
        displayNum -= 1;
        if displayNum <= 0:
            break;
        
'''
    note: the deduplicate function can run when there are no more than 100M record
'''  
def deduplicate(outFile, inFiles):
    import md5;
    for file in inFiles:
        succ = file.next();
        urlset = set();
        while succ <> -1:
            if succ >= 0:
                element = file.getCurrElement();
                url = element.url;
                digest = md5.md5(url).digest();
                if not urlset.__contains__(digest):
                    outFile.write(element.__str__() + '\n');
                succ = file.next();
                    
def toTREC(inFile, outFile):
    while inFile.next() <> -1:
        element = inFile.getCurrElement();
#        print element;
        outFile.write(element.toTRECFormat() + '\n');
    inFile.close();
    outFile.close();           
           
            
if __name__ == '__main__':
    import sys;
    if sys.argv[1] == '--display-profile' or sys.argv[1] == '--deduplicate-profile':
        fileFactory = ProfileFileFactory();
        file = ProfileFile();
    elif sys.argv[1] == '--display-posting' or sys.argv[1] == '--deduplicate-posting':
        fileFactory = PostingFileFactory();
        file = PostingFile();
    if sys.argv[1].startswith('--display'):
        file = fileFactory.createFile();
        file.open(sys.argv[2]);
        display(file, int(sys.argv[3]));
    elif sys.argv[1].startswith('--deduplicate'):
        outFile = open(sys.argv[3], 'w');
        file = open(sys.argv[2]);
        lines = file.readlines();
        inFiles = [];
        for line in lines:
            inFile = fileFactory.createFile();
            inFile.open(line.strip());
            inFiles.append(inFile);
        deduplicate(outFile, inFiles);
        for inFile in inFiles:
            inFile.close();
    elif sys.argv[1].startswith('--totrec'):
        outFile = open(sys.argv[3], 'w');
        inFile = PostingFile();
        inFile.open(sys.argv[2]);
        toTREC(inFile, outFile);
    
                   
